package com.leiy.job.task;

import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.JsonPathSelector;
import us.codecraft.webmagic.selector.Selectable;

import java.text.MessageFormat;
import java.util.List;

@Component
public class JobProcessor implements PageProcessor {
    private String url = "https://search.51job.com/list/040000,000000,0000,32%252c01,9,99,java,2,{0}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
    private static final int pageSize = 10;
    @Scheduled(initialDelay = 1000, fixedDelay = 100*1000)
    public void process(){
        // Spider spider =
        Spider spider = Spider.create(new JobProcessor())
                .addUrl(MessageFormat.format(url, "1"))
                .setScheduler(new QueueScheduler()
                        .setDuplicateRemover(
                                new BloomFilterDuplicateRemover(10*0000)
                        ))
                .thread(5);
                //.run();
        spider.run();
    }

    private Site site = Site.me()
            .setCharset("gbk") // utf8  因为51job的页面时jbk的编码，需要匹配
            //.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")
            .setTimeOut(10*1000) // 超时
            .setRetrySleepTime(3*1000) // 重试的间隔时间
            .setRetryTimes(3); // 重试次数

    @Override
    public Site getSite() {
        return site;
    }
    @Override
    public void process(Page page) {

        // 因为数据在<script>标签里
        String dataJson = page.getHtml().xpath("//script").regex(".*engine_search_result.*").get();
        if(dataJson != null && dataJson.trim().length() > 0){
            // 去除json数据之外的 内容
            dataJson = dataJson.substring(dataJson.indexOf('{'));
            dataJson = dataJson.substring(0,dataJson.lastIndexOf('}')+1);
            // json表达式，获取源数据
            JsonPathSelector jsonPathSelector = new JsonPathSelector("$.engine_search_result[*].job_href");
            List<String> list = jsonPathSelector.selectList(dataJson);
            System.out.println(list.size());
            for (String jobInfoUrl : list) {
                page.addTargetRequest(jobInfoUrl);
            }
            // 下一页
        }else{
            // 如果为空，表示这是招聘详情页，解析页面，获取招聘详情信息，保存数据
            this.saveJobInfo(page);
        }
        // 下一页 MessageFormat.format("http://www.{0}.com", "baidu")
        for (int i = 2; i < pageSize; i++) {
            String nextPageUrl =  MessageFormat.format(url, i);
            page.addTargetRequest(nextPageUrl);
        }


        // 解析页面，获取招聘信息详情的url地址
       /* List<Selectable> nodes = page.getHtml().css("div.rjlist div.e").nodes();
        System.out.println(nodes.size());
        // 判断获取到的集合是否为空
        if(nodes.size()==0){
            // 如果为空，表示这是招聘详情页，解析页面，获取招聘详情信息，保存数据
            this.saveJobInfo(page);
        }else {
            // 如果不为空，表示这是列表页，解析出详情页的url地址，放到任务队列中
            for (Selectable node : nodes) {
                String jobInfoUrl = node.links().toString();
                System.out.println(jobInfoUrl);
                page.addTargetRequest(jobInfoUrl);
            }
            // 下一页
        }*/


    }
    // 解析页面，获取招聘详情信息，保存数据
    private void saveJobInfo(Page page) {
    }


}
